1:- module('tokenizer', [tokenize/2]).    2:- set_prolog_flag(double_quotes,chars).    3:- initialization(main).    4:- include(common_grammar).    5
    6tokenize(Text,Output) :- phrase(tokenize(Output),Text).
    7tokenize([A]) --> string_literal(A).
    8tokenize([A]) --> an_int(A).
    9tokenize([A]) --> symbol(A).
   10tokenize([";"]) --> ";".
   11tokenize(["+="]) --> "+=".
   12tokenize(["*="]) --> "*=".
   13tokenize(["+"]) --> "+".
   14tokenize(["-"]) --> "-".
   15tokenize([">"]) --> ">".
   16tokenize(["<"]) --> "<".
   17tokenize([")"]) --> ")".
   18tokenize(["("]) --> "(".
   19tokenize(["!"]) --> "!".
   20tokenize(["@"]) --> "@".
   21tokenize(["="]) --> "=".
   22tokenize(["*"]) --> "*".
   23tokenize(["/"]) --> "/".
   24tokenize(["|"]) --> "|".
   25tokenize(["&"]) --> "&".
   26tokenize(["."]) --> ".".
   27tokenize(["."]) --> "?".
   28tokenize(["^"]) --> "^".
   29tokenize(["?"]) --> "?".
   30tokenize(["$"]) --> "$".
   31tokenize(["#"]) --> "#".
   32tokenize(["%"]) --> "%".
   33tokenize([A|B]) --> tokenize([A]),skip_space,tokenize(B).
   34skip_space -->
   35  [C], {code_type(C, space)}, skip_space.
   36skip_space --> []